from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
import numpy as np
from python_ai.common.xcommon import sep

pd.set_option('display.max_columns', None)

X = ['我 爱 你',
     '我 恨 你 恨 你',
     '我 apple is fruit',
     '我 vehicle is matter']
m = len(X)

# count = CountVectorizer(token_pattern='[a-zA-Z\u4e00­\u9fa5]+')  # ATTENTION - is ord AD
count = CountVectorizer(token_pattern='[a-zA-Z\u4e00-\u9fa5]+')  # ATTENTION - is ord 2D

print(f'{ord("­"):04x} vs {ord("-"):04x}')
print(f'{ord("­")} vs {ord("-")}')
print(f'{chr(173)}, {chr(45)}')

x = count.fit_transform(X)
print(x.A)
print(x.toarray())
print(count.get_feature_names())

sep('tf in pandas DataFrame')
tf = pd.DataFrame(x.A, columns=count.get_feature_names())
print(tf)
sep('df(d, t)')
print(tf.sum(axis=0))
sep('idf(t)')
print(tf.shape[0])
idf = 1 + np.log((1 + m)/(1 + (tf != 0).sum(axis=0)))
print(idf)

from sklearn.feature_extraction.text import TfidfVectorizer
ti = TfidfVectorizer(norm=None,
                     # token_pattern='[a-zA-Z\u4e00­\u9fa5]+',  # ATTENTION - is ord AD
                     token_pattern='[a-zA-Z\u4e00-\u9fa5]+',  # ATTENTION - is ord 2D
                     )
x2 = ti.fit_transform(X)
print(ti.get_feature_names())
print(x2.A)

print(x2.A.shape, x.A.shape, idf.shape)
tf_idf = tf * idf
print(tf_idf)
print(np.unique(np.equal(x2.A, tf_idf)))

from python_ai.ML_2.bayes.follow_teacher.x_tf_idf_deeper_again import TfidfMyImpl
tf_idf_model = TfidfMyImpl('[a-zA-Z\u4e00-\u9fa5]+')
x3 = tf_idf_model.fit_transform(X)
print(tf_idf_model.get_feature_names())
print(x3)
print(np.unique(np.equal(x2.A, tf_idf)))
